{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Logistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import module\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read FE_csv\n",
    "train = pd.read_csv('HOME_WORK_FE_pima-indians-diabetes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "      <th>Target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.860433</td>\n",
       "      <td>0.122805</td>\n",
       "      <td>0.915180</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>0.200192</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.154909</td>\n",
       "      <td>-0.243335</td>\n",
       "      <td>0.351906</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>-0.737476</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>1.980068</td>\n",
       "      <td>-0.365382</td>\n",
       "      <td>-0.873008</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>-1.179520</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.026951</td>\n",
       "      <td>-0.243335</td>\n",
       "      <td>-0.211369</td>\n",
       "      <td>-0.480077</td>\n",
       "      <td>-0.536547</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.508548</td>\n",
       "      <td>-1.829943</td>\n",
       "      <td>0.915180</td>\n",
       "      <td>0.367368</td>\n",
       "      <td>1.472741</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.860433        0.122805   \n",
       "1  -0.844885                     -1.154909       -0.243335   \n",
       "2   1.233880                      1.980068       -0.365382   \n",
       "3  -0.844885                     -1.026951       -0.243335   \n",
       "4  -1.141852                      0.508548       -1.829943   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.915180      -0.236791  0.200192   \n",
       "1                     0.351906      -0.236791 -0.737476   \n",
       "2                    -0.873008      -0.236791 -1.179520   \n",
       "3                    -0.211369      -0.480077 -0.536547   \n",
       "4                     0.915180       0.367368  1.472741   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  Target  \n",
       "0                    0.468492  1.425995       1  \n",
       "1                   -0.365061 -0.190672       0  \n",
       "2                    0.604397 -0.105584       1  \n",
       "3                   -0.920763 -1.041549       0  \n",
       "4                    5.484909 -0.020496       1  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#show 5 row\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "pregnants                       768 non-null float64\n",
      "Plasma_glucose_concentration    768 non-null float64\n",
      "blood_pressure                  768 non-null float64\n",
      "Triceps_skin_fold_thickness     768 non-null float64\n",
      "serum_insulin                   768 non-null float64\n",
      "BMI                             768 non-null float64\n",
      "Diabetes_pedigree_function      768 non-null float64\n",
      "Age                             768 non-null float64\n",
      "Target                          768 non-null int64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "#train info  没有缺失值\n",
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1\n",
       "1    0\n",
       "2    1\n",
       "3    0\n",
       "4    1\n",
       "Name: Target, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#prepare data \n",
    "X_train = train.drop(['Target'],axis=1)\n",
    "y_train = train['Target']\n",
    "\n",
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pregnants</th>\n",
       "      <th>Plasma_glucose_concentration</th>\n",
       "      <th>blood_pressure</th>\n",
       "      <th>Triceps_skin_fold_thickness</th>\n",
       "      <th>serum_insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>Diabetes_pedigree_function</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.860433</td>\n",
       "      <td>0.122805</td>\n",
       "      <td>0.915180</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>0.200192</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.154909</td>\n",
       "      <td>-0.243335</td>\n",
       "      <td>0.351906</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>-0.737476</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>1.980068</td>\n",
       "      <td>-0.365382</td>\n",
       "      <td>-0.873008</td>\n",
       "      <td>-0.236791</td>\n",
       "      <td>-1.179520</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.026951</td>\n",
       "      <td>-0.243335</td>\n",
       "      <td>-0.211369</td>\n",
       "      <td>-0.480077</td>\n",
       "      <td>-0.536547</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.508548</td>\n",
       "      <td>-1.829943</td>\n",
       "      <td>0.915180</td>\n",
       "      <td>0.367368</td>\n",
       "      <td>1.472741</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pregnants  Plasma_glucose_concentration  blood_pressure  \\\n",
       "0   0.639947                      0.860433        0.122805   \n",
       "1  -0.844885                     -1.154909       -0.243335   \n",
       "2   1.233880                      1.980068       -0.365382   \n",
       "3  -0.844885                     -1.026951       -0.243335   \n",
       "4  -1.141852                      0.508548       -1.829943   \n",
       "\n",
       "   Triceps_skin_fold_thickness  serum_insulin       BMI  \\\n",
       "0                     0.915180      -0.236791  0.200192   \n",
       "1                     0.351906      -0.236791 -0.737476   \n",
       "2                    -0.873008      -0.236791 -1.179520   \n",
       "3                    -0.211369      -0.480077 -0.536547   \n",
       "4                     0.915180       0.367368  1.472741   \n",
       "\n",
       "   Diabetes_pedigree_function       Age  \n",
       "0                    0.468492  1.425995  \n",
       "1                   -0.365061 -0.190672  \n",
       "2                    0.604397 -0.105584  \n",
       "3                   -0.920763 -1.041549  \n",
       "4                    5.484909 -0.020496  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['pregnants', 'Plasma_glucose_concentration', 'blood_pressure',\n",
      "       'Triceps_skin_fold_thickness', 'serum_insulin', 'BMI',\n",
      "       'Diabetes_pedigree_function', 'Age'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "print(feat_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 默认参数的Logistic Regression  评分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()#默认是L2正则，C=1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is:  [0.49149594 0.51857461 0.47502566 0.42942148 0.48606183]\n",
      "cv logloss is: 0.48011590288151973\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优（模型选择）\n",
    "#分类任务中交叉验证缺省是采用StratifiedKFold\n",
    "#采用5折交叉验证，用log似然损失\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "#%timeit loss_sparse = cross_val_score(lr, X_train_sparse, y_train, cv=3, scoring='neg_log_loss')\n",
    "print('logloss of each fold is: ',-loss)\n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy of each fold is:  [0.75974026 0.73376623 0.75974026 0.79738562 0.77124183]\n",
      "cv accuracy is: 0.7643748408454292\n"
     ]
    }
   ],
   "source": [
    "# https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html#sklearn.metrics.accuracy_score\n",
    "accuracy = cross_val_score(lr, X_train, y_train, cv=5, scoring='accuracy')\n",
    "print('accuracy of each fold is: ',accuracy)\n",
    "print('cv accuracy is:', accuracy.mean())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 使用GridSeartchCV进行参数调优\n",
    "#采用5折交叉验证，分别用log似然损失和正确率，对Logistic回归模型的正则超参数调优\n",
    "\n",
    "ogistic回归的需要调整超参数有：C（正则系数，一般在log域（取log后的值）均匀设置候选参数）和正则函数penalty（L2/L1） 目标函数为：J = C* sum(logloss(f(xi), yi)) +* penalty\n",
    "\n",
    "在sklearn框架下，不同学习器的参数调整步骤相同： 设置参数搜索范围 生成GridSearchCV的实例（参数） 调用GridSearchCV的fit方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# 需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.47979410207912676\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)\n",
    "#C = 1是最优参数了，所以不需要再更改C的范围了\n",
    "#最优的惩罚项是L1正则\n",
    "#分数为0.47979410207912676，好于之前的#cv logloss is: 0.48011590288151973"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=0)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# 需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid_1= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='accuracy')\n",
    "grid_1.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7669270833333334\n",
      "{'C': 100, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(grid_1.best_score_)\n",
    "print(grid_1.best_params_)\n",
    "#0.7669270833333334,略好于之前的分数cv accuracy is: 0.7643748408454292"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VOXZ//HPNVkIYQ0QlCURRFQWBSEJKmrdpS7ghkD71KVWbCvVp7/W1m5a8bFP+/RZbCtWcataC7hLlRasYq0oQtiURQVxIYCAIJss2a7fH3OCESEzWSZnZvJ9v17nlZkz58x8T6Jzcc597vs2d0dERKQukbADiIhI8lOxEBGRmFQsREQkJhULERGJScVCRERiUrEQEZGYVCxERCQmFQsREYlJxUJERGLKDDtAU+nSpYv36tUr7BgiIillwYIFn7h7fqztEloszGwE8DsgA7jP3X+93+v/B5wWPM0Furp7x+C1K4CfB6/9h7s/VNdn9erVi9LS0qaMLyKS9szsw3i2S1ixMLMMYBJwFlAGzDez6e6+vGYbd/9+re2/BxwXPO4E3AIUAQ4sCPb9NFF5RUTk4BLZZlECrHL31e5eDkwFRtWx/ThgSvD4HOAFd98SFIgXgBEJzCoiInVIZLHoAayp9bwsWPclZnYY0Bt4qb77iohI4iVLA/dY4Al3r6rPTmY2HhgPUFhYmIhcIpKiKioqKCsrY8+ePWFHSQo5OTn07NmTrKysBu2fyGKxFiio9bxnsO5AxgLX7bfvqfvt+/L+O7n7ZGAyQFFRkSbmEJF9ysrKaNeuHb169cLMwo4TKndn8+bNlJWV0bt37wa9RyIvQ80H+ppZbzPLJloQpu+/kZkdDeQBr9daPRM428zyzCwPODtYJyISlz179tC5c+cWXygAzIzOnTs36iwrYWcW7l5pZhOIfslnAA+4+zIzmwiUuntN4RgLTPVaU/a5+xYzu41owQGY6O5bEpVVRNKTCsXnGvu7SGibhbvPAGbst+7m/Z7/8iD7PgA8kLBwgepq59d/f5tvHH8YBZ1yE/1xIpLExtwTvcAx7doTQk6SfFr8cB8fbP6MqfM+YuSdr/Lae5+EHUdE0kjbtm33PR4xYgQdO3bk/PPPP+C21113HYMHD6Z///60bt2awYMHM3jwYJ544ol6febChQv5+9//3qjcB9Lii8Xh+W15dsJJdG7bim/cP4+HX/+AWlfERESaxI033sgjjzxy0NcnTZrE4sWLmTFjBn369GHx4sUsXryYSy+9tF6fo2KRQL27tOHp757IqUfmc/Ozy/jp029RXlkddiwRSSNnnHEG7dq1a9C+K1eu5JxzzmHo0KGccsopvPvuuwBMnTqVgQMHMmjQIE477TR2797NxIkTefTRRxt0VlKXZOlnEbp2OVlMvryI/5n1Dne9/B4rN+zkj/82lPx2rcKOJiKNdOtfl7F83faY2y1fH92mpu2iLv27t+eWCwY0Ols8xo8fz3333UefPn2YM2cOEyZMYNasWdx66628/PLLHHLIIWzdupXWrVtz8803s3TpUu64444mzaBiUUtGxPjRiKPp1609Nz6xhFF3vsrky4sY2KND2NFEpIXaunUrc+fO5ZJLLtm3rrKyEoDhw4dz+eWXM3r0aC6++OKE5lCxOIALBnWnd5c2jH+4lEvvfo3/unQQIwd1DzuWiDRQvGcAyXg3lLvTpUsXFi9e/KXX7r33Xt544w2ee+45hgwZwqJFixKWQ20WBzGwRweenXASx/TowPVTFvGbv79NVbUavkWkeeXl5dGtWzeefvppAKqrq1myZAkAq1ev5vjjj+e2224jLy+PtWvX0q5dO3bs2NHkOVQs6pDfrhWPfut4xpUU8MeX3+Oah0vZvqci7FgikoJOPvlkRo8ezYsvvkjPnj2ZOTP+QSmmTp3K3XffzaBBgxgwYADPPfccAN///vc55phjOOaYYzjttNMYOHAgp59+OkuWLOG4445r0gZuS5fbRIuKijxRkx+5O4/M/ZBb/7qcXp1zue+KYnp3aZOQzxKRprFixQr69etXr32S8TJUUzrQ78TMFrh7Uax9dWYRBzPj8hN68eerh7Hls3JG3fkq/3x3U9ixRKSJTbv2hLQtFI2lYlEPJ/TpzPQJJ9G9Y2uuenAe976yWh34RKRFULGop4JOuTz5nRM5u/+h3D5jBT94bAl7Kuo1DYeISMpRsWiANq0yuevrQ/j+mUfy1KK1jJk8lw3bNcGKiKQvFYsGikSMG87sy93/NpSVG3ZwwR9eZdFHn4YdS0QkIVQsGmnEwEN56rsn0iorwph75vLEgrKwI4lIQz14XnSRL1GxaAJHH9qe6dedRFGvPH74+BJue245lVUaiFCkpasZonzx4sWccMIJDBgwgGOPPZZp06Z9adtkH6Jcw300kbw22Tz0zRJuf34F97/6Pu9u2MGd44bQIbdhk6OLSPrIzc3l4Ycfpm/fvqxbt46hQ4dyzjnn0LFjx33bTJo0CYAPPviA888//4DDe8Rj4cKFLF26lBEjRjRJ9ho6s2hCWRkRfjlyAL+55Bjmrt7MqEmvsnJD03e7F5HUcuSRR9K3b18AunfvTteuXdm0Kf6+WhqiPE2NKS6kT35bvv3nBVx012vcMWYwZ/Y/JOxYIi3X326Cj9+Kvd3Hb0Z/xtNucegx8NVf1zvKvHnzKC8vp0+fPnHvkwxDlOvMIkGKenVi+oST6NUll2seKWXS7FXqwCfSwq1fv55vfOMbPPjgg0Qi8X391h6ifPDgwVx33XWsW7cO+HyI8vvuu4/q6sS2k+rMIoG6d2zN49eeyI+ffJPfznyH5eu389tLjyU3W792kWYV7xlAzRnFVc83eYTt27dz3nnncfvtt3P88cfHvZ+GKG8hWmdn8Luxg/nxiKOZ8dZ6Lv3j66zdujvsWCLSjMrLy7nooou4/PLL6z2ntoYob0HMjO+c2of7ryhizZZdjPzDq8x7f0vYsUSkmTz22GO88sor/OlPf9p3S2x97nbSEOVNKJFDlDelVRt3Mv7hUtZ8uotbRw7ka8MKw44kkpYaMkR5Ii9DJQMNUZ5CjujalqevG86Jfbrw06ff4hfPLKVCHfhEksNVz6dtoWgsFYsQdGidxQNXFnPtKYfzyNwP+bf73mDzzr1hxxIROSgVi5BkRIyfnNuP/xsziEVrtjLyzjksX7c97FgiaSVdLrM3hcb+LlQsQnbRcT15/NoTqKyu5pI/vsbf3lofdiSRtJCTk8PmzZtVMIgWis2bN5OTk9Pg91ADd5LYuH0P1/55AYs+2sr1px/Bv595JJGIhR1LJGVVVFRQVlbGnj2aawaixbNnz55kZX1xvLp4G7jVOyxJdG2fw9Txx/Ozp5fy+5dW8fbHO/jfMYNp20p/IpGGyMrKonfv3mHHSBu6DJVEWmVm8NtLj+UX5/fnHys2cPFdc/hw82dhxxIRUbFINmbG1Sf15uFvDmPD9r2MmjSHOas+CTuWiLRwCS0WZjbCzN4xs1VmdtNBtrnMzJab2TIz+0ut9VVmtjhYpicyZzI6qW8Xnr1uOPltW3H5A/N4cM77aqgTkdAk7IK4mWUAk4CzgDJgvplNd/fltbbpC/wEGO7un5pZ11pvsdvdBycqXyro1aUNT333RL4/bQm3/nU5b6/fwcQLB9AqMyPsaCLSwiTyzKIEWOXuq929HJgKjNpvm2uASe7+KYC7b0xgnpTULieLyd8YyvdOP4JppWv42r1vsHGH7u4QkeaVyGLRA1hT63lZsK62I4EjzWyOmc01s9rzAOaYWWmw/sIDfYCZjQ+2Ka3PrFOpJhIxfnD2UUz62hCWr9vOqDvn8GbZ1rBjiUgLEnYDdybQFzgVGAfca2Y1k9IeFtz7+zXgDjP70rRS7j7Z3YvcvSg/P7+5MofmvGO78cR3TiBixui7X+fZxWvDjiQiLUQii8VaoKDW857ButrKgOnuXuHu7wPvEi0euPva4Odq4GXguARmTRkDundg+oThDCroyA1TF/Off1tBVbUavkUksRJZLOYDfc2st5llA2OB/e9qeoboWQVm1oXoZanVZpZnZq1qrR8OLEcA6Ny2FX++ehhfH1bIPf9czdUPzWfb7oqwY4lIGkvY3VDuXmlmE4CZQAbwgLsvM7OJQKm7Tw9eO9vMlgNVwI3uvtnMTgTuMbNqogXt17XvohLIzoxw+0XH0K9be345fRkX3TWHttmZtM7OYNq1J4QdT0TSjMaGSgNvrN7Mdx5dyLbdFRyR34aZ3/9K2JFEJEVo8qMWZNjhnZk+YTitMiO8s2EnqzbuDDuSiKQZFYs00TMvl6MPbYcBU+d9FHYcEUkzKhZpJCsjQsfcLJ5cWMbeyqqw44hIGlGxSCPTrj2BO8Yex6e7Kpi1bEPYcUQkjahYpJmTj+hCj46tmTZ/TeyNRUTipGIB8OB50SUNRCLGZUUFvLrqEz7avCvsOCKSJlQs0tBlxT2JGEwrVUO3iDQNFQuAqkpIk/4mAN06tObUo7ryeGkZlVXVYccRkTSgYrH5PVhXCp+l1+joY4sL2LhjL7PfSd/ReEWk+ahYdDocMnNg64dQnj7zXZ92dFfy27VSnwsRaRIqFmaQ1xuqyuH1SWGnaTJZGRFGD+3J7Hc2sn7b7rDjiEiKU7EAyOkAuZ3h1TtgR/r0TxhTXEC1w+OlZWFHEZEUp2JRo2MvqNoLs28PO0mTOaxzG4Yf0Zlp89dQrTkvRKQRVCwArnoexr8ExdfAokdgQ/qMhj6muJC1W3fz6qpPwo4iIilMxaK2r/wIstvBCzeHnaTJnDPgEPJys5g6Xw3dItJwKha15XaCU34Iq16A914KO02TaJWZwcVDevLC8g18snNv2HFEJEWpWOxv2LXQsRBm/QKq02Pk1rHFBVRUOU8tVEO3iDSMisX+MlvBmb+EDUthyZSw0zSJvoe0Y+hheUydv4Z0mRlRRJqXisWBDLgYehTBi7elTUe9scUFrN70GfPe3xJ2FBFJQSoWB2IG59wOOz+G1+4MO02TOO/YbrRrlamhy0WkQVQsDqbweOg3Eub8DnZ8HHaaRsvNzmTUcd15/q31bNtVEXYcEUkxKhZ1OfOX0WFA0qSj3tjiQvZWVvPM4rVhRxGRFKNiUZfOfaDkGlj0Z9iwLOw0jTawRwcG9mjPlHkfqaFbROpFxSKWU26EVunTUW9scSFvf7yDN8u2hR1FRFKIikUsuZ2iBWPVP2DVi2GnabRRg7vTOitDPbpFpF5ULOJRMh46HpYWHfXa5WRx3rHdmL54HZ/trQw7joikCBWLeNR01Nu4DBb/Jew0jTaupIDPyqv465J1YUcRkRShYhGvARdBz2J46T9g786w0zTKkMI8+nZty1T1uRCROKlYxMsMzg466r2e2h31zIwxxQUsXrOVtz/eHnYcEUkBKhb1UTgM+o9Ki456Fw/pSXZGhKnzdHYhIrGpWNTXmb+Eqoro5agU1qlNNucMPJSnFpaxpyK1G+1FJPFULOqr0+HRu6MW/Rk+Xhp2mkYZV1zA9j2V/H1pap8liUjiJbRYmNkIM3vHzFaZ2U0H2eYyM1tuZsvM7C+11l9hZiuD5YpE5qy3U34IOe1TvqPe8Yd3prBTLlPmqc+FiNQtYcXCzDKAScBXgf7AODPrv982fYGfAMPdfQDw78H6TsAtwDCgBLjFzPISlbXecjvBKT+C916MdtZLUZFItKH7jfe3sHpTat/hJSKJlcgzixJglbuvdvdyYCowar9trgEmufunAO6+MVh/DvCCu28JXnsBGJHArPVXcg3k9Ur5jnqjh/YkI2JMK1VDt4gcXCKLRQ+g9jdQWbCutiOBI81sjpnNNbMR9dgXMxtvZqVmVrpp06YmjB6HfR31lsPiR5v3s5tQ1/Y5nHF0V55cUEZ5ZXXYcUQkSYXdwJ0J9AVOBcYB95pZx3h3dvfJ7l7k7kX5+fkJiliH/hdCz5KU76g3tqSAT3aW8+KKDWFHEZEklchisRYoqPW8Z7CutjJgurtXuPv7wLtEi0c8+4Zv34x6G+C1P4SdpsG+cmRXunXIYYp6dIvIQSSyWMwH+ppZbzPLBsYC0/fb5hmiZxWYWReil6VWAzOBs80sL2jYPjtYl3wKSqJnGK/9HravDztNg2REjNFFBfxr5SbKPt0VdhwRSUIJKxbuXglMIPolvwJ4zN2XmdlEMxsZbDYT2Gxmy4HZwI3uvtndtwC3ES0484GJwbrkdOYt0Y56s1O3o95lRT0BeKy0LOQkIpKMLF1mTCsqKvLS0tLwAsz8Gbw+Cb79Khw6MLwcjXD5A/NYuWEHr/74dDIiFnYcEWkGZrbA3YtibRd2A3f6OPkHkNMBXvhF2EkabFxxAeu37eGf726MvbGItCgqFk0ltxN85Ufw3kuwMjU76p3R7xC6tM3W4IIi8iX1LhZmFjGz9okIk/KKr4G83jDr51CVerPQZWdGuGRoT158eyMbt+8JO46IJJG4ioWZ/cXM2ptZG2ApsNzMbkxstBSUmR3tqLdpRcp21BtTVEBVtfP4AjV0i8jn4j2z6O/u24ELgb8BvYFvJCxVKus/CgqGwezbU7Kj3uH5bRnWuxPT5q+hujo9bn4QkcaLt1hkmVkW0WIx3d0rAH2THMi+GfU2RPtepKBxJYV8tGUXc1dvDjuKiCSJeIvFPcAHQBvgFTM7DNB8nAdTUByds3vO72H7urDT1NuIgYfSoXWWenSLyD5xFQt3/72793D3cz3qQ+C0BGdLbWfcAtWV8NLtYSept5ysDC46rgczl37Mls/Kw44jIkkg3gbuG4IGbjOz+81sIXB6grOltk69Ydi10Ybuj98KO029jS0poLyqmqcWqqFbROK/DPXNoIH7bCCPaOP2rxOWKl2c8sNoR71ZP4cU6yl/9KHtGVzQkWnz15AuvfxFpOHiLRY1Yz+cCzzi7stqrZODaZ0HX/kxrH45JWfUG1dSwMqNO1n40adhRxGRkMVbLBaY2SyixWKmmbUDNFNOPIq/lbId9c4/tjttsjOYoh7dIi1evMXiauAmoNjddwHZwFUJS5VOMrPhrFth09uw+M9hp6mXNq0yGTm4O8+/uZ7teyrCjiMiIYr3bqhqohMQ/dzM/hs40d3fTGiydNJvZLSj3ku3w94dYaepl7HFheyuqGL64tS7BVhEmk68d0P9GrgBWB4s15vZrxIZLK3UdNT7bGO070UKObZnB/p1a8/U+R+FHUVEQhTvZahzgbPc/QF3fwAYAZyfuFhpqKAYBlwcnX41hTrqmRljiwtYunY7S9duCzuOiISkPqPOdqz1uENTB2kRzrwFvApeSq0Z9S4c3INWmRGdXYi0YPEWi/8EFpnZn8zsIWABkHpdk8OW1wtKxsPiv8D61Gny6ZCbxXnHdOPZRevYVZ5ad3SJSNOIt4F7CnA88BTwJHCCu09LZLC0dcoPoXXHlOuoN7akkB17K3n+zfVhRxGRENRZLMxsSM0CdAPKgqV7sE7qq6aj3vv/hJUvhJ0mbsW98jg8vw1TNbigSIuUGeP1/6njNUfjQzVM0dUwb3J0vu4+p0NGrD9D+Goaun81421WbthB30PahR1JRJpRnWcW7n5aHYsKRUNlZsOZQUe9RY+EnSZulwzpSVaG6exCpAWKt5/FxQdYzjCzrokOmLb6XQAFxwcz6qVGR73ObVtxdv9DeWphGXsrq8KOIyLNqD7DfdwHfD1Y7gV+DMwxM02v2hBmcM7t8NkmmPO7sNPEbUxxAZ/uqmDmsg1hRxGRZhRvscgE+rn7Je5+CdCfaJvFMKJFQxqiZxEMvAReuxO2rQ07TVxOOqILPfNaM019LkRalHiLRYG71/6n5MZg3RZAI8w1xhk3p1RHvUjEGFNUwJxVm/lw82dhxxGRZhJvsXjZzJ4zsyvM7ApgerCuDbA1cfFagLxe0Rn1lkyB9UvCThOX0UUFRAymqaFbpMWIt1hcBzwIDA6Wh4Dr3P0zd9dc3I11cmp11Du0Qw6nHdWVxxeUUVGlaU1EWoJ4e3A78CrwEvAi8Iprrs2m07ojfOUmeP8VWDkr7DRxGVtSyKYde5n99sawo4hIM4j31tnLgHnApcBlwBtmdmkig7U4Rd+ETofDrF+kxIx6px2VT9d2rdTnQqSFiPcy1M+IzpJ3hbtfDpQAv0hcrBYoMxvOmgifvAOLHg47TUyZGRFGF/Xk5Xc2sn7b7rDjiEiCxVssIu5e+3rD5nj2NbMRZvaOma0ys5sO8PqVZrbJzBYHy7dqvVZVa/30OHOmtqPPh8ITYPavYM/2sNPENKaokGqHx0vLwo4iIgkWb7H4u5nNDL7crwSeB2bUtYOZZQCTgK8S7Zcxzsz6H2DTae4+OFjuq7V+d631I+PMmdr2zaiXGh31CjvnctIRXZg2fw3V1WrCEkln8TZw3whMBo4NlsnuHqszXgmwyt1Xu3s5MBUY1ZiwLULPoTDwUng9NTrqjS0pYO3W3fxr1SdhRxGRBIp7pjx3f9Ld/1+wPB3HLj2A2q2fZcG6/V1iZm+a2RNmVlBrfY6ZlZrZXDO7MN6caeGMm8Gr4aXbwk4S01n9DyEvN4up89SjWySdxZrPYoeZbT/AssPMmuKi+l+BXu5+LPAC0f4bNQ5z9yLga8AdZtbnAPnGBwWldNOmTU0QJ0nkHQbDvg1LpsK6xWGnqVOrzAwuGdKTF5Zv4JOde8OOIyIJEmuI8nbu3v4ASzt3bx/jvdcCtc8Uegbrar//Znev+Ya5Dxha67W1wc/VwMvAcQfIN9ndi9y9KD8/P0acFHPyD6ITJaVAR72xJQVUVjtPLlBDt0i6ivsyVAPMB/qaWW8zywbGEh0mZB8z61br6UhgRbA+z8xaBY+7AMOB5QnMmnxad4RTb4IP/gXvzgw7TZ2O6NqO4l55TJu/BvXVFElPCSsW7l4JTABmEi0Cj7n7MjObaGY1dzddb2bLzGwJcD1wZbC+H1AarJ8N/NrdW1axABh6FXTqE51RL8k76o0pLmT1J5/xxvtbwo4iIglg6fIvwaKiIi8tLQ07RtNb8RxM+zqc979QfHXYaQ5qd3kVJb/6B2f2O4T/GzM47DgiEiczWxC0D9cpkZehpCkcfR4Unpj0HfVaZ2dw4eAezHhrPdt2adR6kXSjYpHszOCc/4Bdn8CcO8JOU6exJQXsrazm6UVq6BZJNyoWqaDHUDhmNLw+CbYl7xfxgO4dOKZHB6aqoVsk7ahYpIrTfxG9hTbJZ9QbW1LA2x/vYEnZtrCjiEgTUrFIFXmHwfHfjs6ol8Qd9UYO6k7rrAz16BZJMyoWqeTkH0DrTgfvqPfgedElRO1ysrhgUDemL1nHzr3JfbuviMRPxSKV5HSAU3+S9B31xhQXsqu8ir8uWRd2FBFpIioWqaboKuh8RNBRLzlvUR1S2JEjD2mrWfRE0oiKRarJyApm1HsXFj4Ue/sQmBljiwtZsmYrK9Ynb98QEYmfikUqOupcOGw4zP7PpO2od/GQHmRnRtTQLZImVCxSkRmcHXTUe/X/wk5zQB1zsxkx4FCeXrSWPRVVYccRkUZSsUhVPYbAMZfB3Ltga3K2DYwtKWD7nkr+tnR92FFEpJFULFLZGcndUe+EwzvTq3MuU+YlZzETkfipWKSyjoVw/HfgzamwblHYab7EzBhTXMi897fw3qadYccRkUZQsUh1J/8/yO0Ms36RlDPqXTK0B5kR4zHdRiuS0lQsUl3tjnq7k2/ioa7tcjijX1eeWFBGeWV12HFEpIFULNLB0CujHfU+/QA8+b6Qx5YUsvmzcv6xYkPYUUSkgVQs0kFNR73K3bAz+b6QT+mbT/cOOUxRnwuRlKVikS6OOhdatYetH8L6JWGn+YKMiDG6qIBXV33Cmi27wo4jIg2gYpEuzKBTH8Bg8mnwws1QnjxfzJcVFwDweKkaukVSkYpFOsluA92HwuCvwZzfwR9PhNX/DDsVAD06tuYrR+bzWGkZlVXJ164iInVTsUg3GZkw6k64fHr0+cMj4dnrYFf4d0qNLS7k4+17+Oe7m8KOIiL1pGKRrg7/Cnz3dRj+77B4CkwqgaVPhdoX44x+XenStpWGLhdJQSoW6SyrNZx1K4yfDe27wxNXwZRxsG1tOHEyIlw6tCcvvb2Rjdv3hJJBRBpGxaIl6DYIvvVSdKTa1S/DpGEw716obv62g7HFBVRVO48vKGv2zxaRhlOxaCkyMuHE70UvTfUcCjN+CA9+FTa906wxenVpwwmHd2bq/I+ork6+4UlE5MBULFqaTr3hG8/AhX+ETW/D3SfBy7+ByvJmizC2pIA1W3bz+urNzfaZItI4KhYtkVn09toJ86HfBfDyr+CeU2DNvGb5+HMGHErH3Cz16BZJISoWLVnbrnDpA/C1x2DvDrj/bJjxo+jjBMrJyuCi43owa9kGtnzWfGc0oXnwvOiSBpb96iSW/eqksGM0WrocBzTfsahYCBx5Dlw3F0rGw7zJMOl4eHdmQj9ybHEh5VXVPLXwwA3dY+55nTH3vJ7QDM1l2fptLFu/LewYIo2iYiFRrdrBuf8FV8+CVm3hL5fBE9+EnYnpQHfUoe04rrAjU+evwQ/Q9+PmzTdy8+YbE/LZIlJ/KhbyRQUlcO2/4NSfwoq/wqRiWPyXhHTmG1dcyKqNO1nw4adN/t4i0rQSWizMbISZvWNmq8zspgO8fqWZbTKzxcHyrVqvXWFmK4PlikTmlP1kZsOpP44WjS5HwjPfgUcuhC3vN+nHnHdsN9q2ytQc3SIpIGHFwswygEnAV4H+wDgz63+ATae5++BguS/YtxNwCzAMKAFuMbO8RGVNG1c9H12aStej4aq/w7n/DWUL4K4T4LU/QFVlk7x9m1aZXDCoO8+/tY7teyqa5D1FJDESeWZRAqxy99XuXg5MBUbFue85wAvuvsXdPwVeAEYkKKfUJRKBkmuiDeCHnwqzfg73nQHr32yStx9XUsCeimqeXbyuSd5PRBIjkcWiB1D7+kJZsG5/l5jZm2b2hJkV1HNfaS4desK4KTD6T7B9LUw+FV64BSp2N+ptj+nRgf7d2jNVfS5EklrYDdx/BXq5+7FEzx4eqs/OZjbezErNrHTTJg17nXCEvp8jAAAM90lEQVRmMOAiuG4eDB4Hc+6Izpnx/iuNeEtjXEkBy9Zt560y3V4qkqwSWSzWAgW1nvcM1u3j7pvdfW/w9D5gaLz7BvtPdvcidy/Kz89vsuASQ24nGDUJLn82epfUQxfAsxNgd8Puaho5uAc5WRGmztfZhUiySmSxmA/0NbPeZpYNjAWm197AzLrVejoSWBE8ngmcbWZ5QcP22cE6SSaHnwrfeQ2G3xC9vfbOElj2TL1vs+3QOotzj+nGs4vXsau8aRrPRaRpJaxYuHslMIHol/wK4DF3X2ZmE81sZLDZ9Wa2zMyWANcDVwb7bgFuI1pw5gMTg3WSbLJz4ayJwZwZ3eDxK2Dq1+o9Z8a4kkJ27q3kuTfXJyioiDRGQtss3H2Gux/p7n3c/fZg3c3uPj14/BN3H+Dug9z9NHd/u9a+D7j7EcHyYCJzShOomTPjrNvgvdnROTPm3xf3nBlFh+XRJ7+NGrpFklTYDdySTjIyYfj18N3XoMcQeP4H8KdzYdO7MXc1M8YWF7Lwo628uyGxAxmKSP2pWEjT63R4tPF71F2wcQXcPRz++V8x58y4eEgPsjKMqerRLZJ0VCwkMczguK9H58w4+nyYfXswZ8b8g+7SuW0rzh5wKE8tKqPcM5oxrIjEomIhidW2K4x+EMZNhb3b4f6z4G8/PuicGeOKC9m6q4LXKo9q5qAiUhcVC2keR30Vvjs3OnTIG/dEx5l6d9aXNjuxT2cKOrVmVvmgEEKKyMGoWEjzyWkP5/4WvjkTsnLhL6Phiau/MGdGJGKMKSpgSVUv1ld3DDGsiNSmYiHNr3AYfPtfcOpPYPmzwZwZU/Z15htdVECEap7aO4w1W3axY0/FASdIEpHmkxl2AGmhMlvBqTdB/wth+vfgmW/Dm9Pggjs4JK8XwzJXMqNiCDP+azYAWRlGx9xsOuVm0zE3i7zcbPLaZJOXm0WnNtnR19pk0TE3m7xgu3Y5mUQiFvKBiqQHFQsJV9ejo5elSu+Hf/wy2pZx2s+4Ied5zqp6k3bn3cbWXRVs2VXO1l3lfPpZ9PHqT3ay5cMKtu4qp7L6wGcdEYO8oLjUFJS83KygyHxeeGq/1jE3mwwVGJEvUbGQ8NXMmXHUV6Md+Wb9jEHWmm4Zmzis8/vQJQMiGWAZ0W2tNUTagmXgFuGzimq273G27q1i655qtu2p4tPd1WzdU8WWXVV8uruKLbv2sHHzTlasqWLz7ip2VwJ8uSiYQfucmgKSFRSU/YpMcAZTs01ebjZZGbqiK+nN0uVacFFRkZeWloYdQxrLHZY9TeUT15BJYgcVdGxfwam2DByjigjVRKgiQpVHqHSj0iNUuFHpRnWt16M/o/uYZWAZGUQimWRkZJCRkUkkM4PMjEwqP/0Iw8nJ7wN4tP3FHYj+9OCxefW+x9Te5gvPq/e17Vjtbfni49qv2RdeI/rcHaP6C8+jv5EvvrfVWqIZqqKvW8OLY81n1PWXif0ede8f69zQa34vlgZnke68790ZNHFhg3Y3swXuXhRrO51ZSHIxg4EXs/LZ35Lje+j99d9Hv6Cqq8CrP/+5b11VdPypLzwPfrp/eV2t97HguXk1keB9sg70PtXV4NVUVVVSXlFBRUUlFZUVVFRWUllZQWVlFVWVFVRVVVFVVUV1VSWVFXuprorunxH8b7Zrw0e1v3qh1uNqItT+eoaar7zgudX8zICa9ca+x9EvvZrtIsG36X6vRT5/7BYJXqv54rd9X5xmERzDzHCL/oQIZsbeLdHe9dmdDqPmEw7+t6xrxeePD/R9XfM7qPv9D/6qWd3vsWfjKgBy8vvW9QkpYc/GlZRbVsI/R8VCklK1ZbDL2kDvk8OOsk8G0DpY4lVeWc0bv/4qVUQ4+ntPE4lAxIwMMyJmnz+PGGbUWp+c/+Jd9quTABhwQ73mKUs6+47juntDTtJ40WNJ/ND+KhYiCZSdGaFT5DMADu2QE3IakYZTq5yIiMSkYiEiIjGpWIiISEwqFiIiEpOKhYiIxKRiISIiMalYiIhITCoWIiISk4qFiIjEpGIhIiIxqViIiEhMKhYiIhKTioWIiMSkUWclKQ3o1iHsCCJSi84sREQkJhULERGJScVCRERiUrEQEZGYEloszGyEmb1jZqvM7KY6trvEzNzMioLnvcxst5ktDpa7E5lTktBVz0cXEUkKCbsbyswygEnAWUAZMN/Mprv78v22awfcALyx31u85+6DE5VPRETil8gzixJglbuvdvdyYCow6gDb3Qb8BtiTwCwiItIIiexn0QNYU+t5GTCs9gZmNgQocPfnzezG/fbvbWaLgO3Az939XwnMKpIwA376atgRmky6HEu6HAc037GE1inPzCLA/wJXHuDl9UChu282s6HAM2Y2wN237/ce44HxAIWFhQlOLCLSciXyMtRaoKDW857BuhrtgIHAy2b2AXA8MN3Mitx9r7tvBnD3BcB7wJH7f4C7T3b3Incvys/PT9BhiIhIIovFfKCvmfU2s2xgLDC95kV33+buXdy9l7v3AuYCI9291MzygwZyzOxwoC+wOoFZRUSkDgm7DOXulWY2AZgJZAAPuPsyM5sIlLr79Dp2PwWYaGYVQDXwbXffkqisIiJSN3P3sDM0iaKiIi8tLQ07hohISjGzBe5eFGs79eAWEZGYVCxERCQmFQsREYlJxUJERGJKmwZuM9sEfNiIt+gCfNJEccKULscBOpZklS7Hki7HAY07lsPcPWZHtbQpFo1lZqXx3BGQ7NLlOEDHkqzS5VjS5TigeY5Fl6FERCQmFQsREYlJxeJzk8MO0ETS5ThAx5Ks0uVY0uU4oBmORW0WIiISk84sREQkJhWLgJndZmZvBnN+zzKz7mFnaigz+62ZvR0cz9Nm1jHsTA1lZqPNbJmZVdfM0Z5K4p2HPhWY2QNmttHMloadpTHMrMDMZpvZ8uC/rRvCztRQZpZjZvPMbElwLLcm7LN0GSrKzNrXTK5kZtcD/d392yHHahAzOxt4KRj59zcA7v7jkGM1iJn1Izry8D3AD909ZUaLDIbZf5da89AD4/afhz5VmNkpwE7gYXcfGHaehjKzbkA3d19oZu2ABcCFqfh3MTMD2rj7TjPLAl4FbnD3uU39WTqzCOw3C18bIGWrqLvPcvfK4OlcohNPpSR3X+Hu74Sdo4HinYc+Jbj7K0DKTxXg7uvdfWHweAewgug00CnHo3YGT7OCJSHfXSoWtZjZ7Wa2Bvg6cHPYeZrIN4G/hR2ihTrQPPQp+aWUrsysF3Ac8Ea4SRrOzDLMbDGwEXjB3RNyLC2qWJjZP8xs6QGWUQDu/jN3LwAeBSaEm7ZusY4l2OZnQCXR40la8RyLSFMzs7bAk8C/73dlIaW4e5W7DyZ6BaHEzBJyiTBhM+UlI3c/M85NHwVmALckME6jxDoWM7sSOB84w5O8Yaoef5dUE2seeglJcH3/SeBRd38q7DxNwd23mtlsYATQ5DchtKgzi7qYWd9aT0cBb4eVpbHMbATwI6Jzmu8KO08LVuc89BKOoFH4fmCFu/9v2Hkaw8zya+52NLPWRG+mSMh3l+6GCpjZk8BRRO+8+ZDovN8p+a9AM1sFtAI2B6vmpvCdXRcBfwDyga3AYnc/J9xU8TOzc4E7+Hwe+ttDjtRgZjYFOJXoCKcbgFvc/f5QQzWAmZ0E/At4i+j/7wA/dfcZ4aVqGDM7FniI6H9fEeAxd5+YkM9SsRARkVh0GUpERGJSsRARkZhULEREJCYVCxERiUnFQkREYlKxEKkHM9sZe6s693/CzA4PHrc1s3vM7D0zW2BmL5vZMDPLNrNXzKxFdZqV5KZiIdJMzGwAkOHuq4NV9xEdmK+vuw8FrgK6BIMOvgiMCSepyJepWIg0gEX9NhjD6i0zGxOsj5jZXcF8Ii+Y2QwzuzTY7evAs8F2fYBhwM/dvRrA3d939+eDbZ8JthdJCjrNFWmYi4HBwCCiPZrnm9krwHCgF9Af6Ep0+OsHgn2GA1OCxwOI9kavOsj7LwWKE5JcpAF0ZiHSMCcBU4IRPzcA/yT65X4S8Li7V7v7x8DsWvt0AzbF8+ZBESkPJucRCZ2KhUjz2Q3kBI+XAYOC2fQOphWwJ+GpROKgYiHSMP8CxgQTz+QDpwDzgDnAJUHbxSFEB96rsQI4AsDd3wNKgVuDUVAxs15mdl7wuDPwibtXNNcBidRFxUKkYZ4G3gSWAC8BPwouOz1JdEa85cCfgYXAtmCf5/li8fgWcAiwysyWAn8iOtsZwGnB9iJJQaPOijQxM2vr7juDs4N5wHB3/ziYb2B28PxgDds17/EUcJO7v9sMkUVi0t1QIk3vuWBCmmzgtuCMA3ffbWa3EJ2H+6OD7RxMlPSMCoUkE51ZiIhITGqzEBGRmFQsREQkJhULERGJScVCRERiUrEQEZGYVCxERCSm/w9+XsgvlsveBAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#pd.DataFrame(grid.cv_results_).to_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#cvresult = pd.DataFrame.from_csv('LogisticGridSearchCV_Otto.csv')\n",
    "#test_means = cv_results['mean_test_score']\n",
    "#test_stds = cv_results['std_test_score'] \n",
    "#train_means = cvresult['mean_train_score']\n",
    "#train_stds = cvresult['std_train_score'] \n",
    "\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    #plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'logloss' )\n",
    "plt.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
